'use strict';

const fs = require('fs');
const url = require('url');
const dayjs = require('dayjs');
const {chromium, firefox} = require('playwright');
const TurndownService = require('turndown');
const got = require('../utils/got');
const cheerio = require('cheerio');
const chinese = require('chinese-simple2traditional');
let Parser = require('../utils/rss/parser');
let parser = new Parser({timeout: 10000});

const turndownService = new TurndownService();

const Service = require('egg').Service;

class CommonSvc extends Service {

  async wangyi(url, boards) {
    let totalTopics = [];

    const tab = 'yutang';
    const author_id = '62f9db67c178251765694c2b';
    const author_name = '网易新闻';
    const author_avatar = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/wangyihao.jpg';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    try {
      const response = await got({
        method: 'get',
        url: url,
      });

      let html = response.data;

      let result = html.match(/window.__INITIAL_STATE__=(.+?);\(function/);
      let json = result[1];
      let objs = JSON.parse(json);

      let data_list;
      if (objs.store.homeArticleList) {
        data_list = objs.store.homeArticleList.data.list;
      } else if (objs.store.domesticArticleList) {
        data_list = objs.store.domesticArticleList.data.list;
      } else if (objs.store.internationalArticleList) {
        data_list = objs.store.internationalArticleList.data.list;
      } else if (objs.store.historyArticleList) {
        data_list = objs.store.historyArticleList.data.list;
      } else if (objs.store.techscienceArticleList) {
        data_list = objs.store.techscienceArticleList.data.list;
      } else if (objs.store.smartArticleList) {
        data_list = objs.store.smartArticleList.data.list;
      } else if (objs.store.tedArticleList) {
        data_list = objs.store.tedArticleList.data.list;
      } else if (objs.store.jpkcArticleList) {
        data_list = objs.store.jpkcArticleList.data.list;
      } else if (objs.store.homeVideoList) {
        data_list = objs.store.homeVideoList.data.list;
      } else {
        console.error(objs);
      }


      for (const data of data_list) {
        let title = data.title;
        let docid;
        let content;
        let pic;
        let visit_count;
        if (url == 'https://m.163.com/touch/video') {
          docid = data.vid;
          content = `https://m.163.com/v/video/${docid}.html`;
          pic = data.cover;
          visit_count = data.playCount || 0;
        } else {
          docid = data.docid;
          content = `https://m.163.com/dy/article/${docid}.html`;
          pic = data.imgsrc;
          visit_count = data.commentCount || 0;
        }


        let topic = {
          title,
          content,
          original_url: content,
          pic,
          tab,
          boards,
          visit_count,
          author_id,
          author_name,
          author_avatar,
          source,
        };
        totalTopics.push(topic);
      }


    } catch (e) {
      this.logger.error(e);
      await this.ctx.helper.dd_warn(`${author_name} ${e.message}`);
    } finally {
      return totalTopics;
    }


  }


  async commonZhihu(loginname, tab) {
    const user = await this.ctx.model.User.findOne({loginname});
    if (!user) {
      throw new Error('commonZhihu参数异常');
    }

    const url = this.ctx.helper.getFromUrl(user.froms, '知乎号');
    if (!url) {
      throw new Error('commonZhihu参数异常');
    }

    const param = {
      url,
      author_info: {
        author_id: user.id,
        author_name: user.loginname,
        author_avatar: user.avatar_url
      },
      topic_info: {
        tab,
        boards: [...user.default_boards]
      },
      selector_info: {
        box: '.List .List-item',
        title: '.ContentItem-title a',
        content: '.ContentItem-title a',
        pic: '.RichContent-cover-inner img',
      },
      scroll: 3000
    }

    const topics = await this.commonFetch(param);

    topics.forEach(v => {
      if (v.pic) {
        v.pic = v.pic.replace('_200x0', '');
      }
    })
    return topics;
  }

  async commonWeixinApi(loginname, tab) {
    const user = await this.ctx.model.User.findOne({loginname});
    if (!user) {
      throw new Error('commonWeixinApi参数异常');
    }

    const url = this.ctx.helper.getFromUrl(user.froms, '公众号');
    if (!url) {
      throw new Error('commonWeixinApi参数异常');
    }

    const param = {
      url,
      author_info: {
        author_id: user.id,
        author_name: user.loginname,
        author_avatar: user.avatar_url
      },
      topic_info: {
        tab,
        boards: [...user.default_boards]
      },
      selector_info: {
        box: 'data.list',
        title: 'title',
        content: 'url',
        intro: 'desc',
      }
    }

    const topics = await this.commonJson(param);
    return topics;
  }


  /**
   * 针对json数据的http爬虫
   */
  async commonJson(param) {
    let topics = [];

    const max_retry_cnt = 3;
    let try_cnt = 0;
    while (topics.length == 0 && try_cnt < max_retry_cnt) {
      try {
        topics = await this._commonJson(param);
      } catch (e) {
        this.logger.warn('commonJson重试', param.author_info.author_name, try_cnt++);
      }
    }

    if (topics.length == 0) {
      await this.ctx.helper.dd_warn('commonJson爬取失败: ' + param.author_info.author_name);
    }

    return topics;
  }


  async _commonJson(param) {
    let totalTopics = [];

    const author_id = param.author_info.author_id;
    const author_name = param.author_info.author_name;
    const author_avatar = param.author_info.author_avatar;
    const source = param.author_info.author_name + dayjs().format('YYYY-MM-DD');

    try {
      const response = await got.get(param.url);

      let jsonData = response.data;

      let items;
      if (param.selector_info.box == '') {
        items = jsonData;
      } else {
        items = eval(`jsonData.${param.selector_info.box}`);
      }

      totalTopics = items
        .map(item => {
          let topic = {
            title: item[param.selector_info.title],
            content: item[param.selector_info.content],
            original_url: item[param.selector_info.content],
            pic: item[param.selector_info.pic],
            intro: item[param.selector_info.intro],
            tags: param.selector_info.tags ? item[param.selector_info.tags] : undefined,
            visit_count: param.selector_info.visit_count ? item[param.selector_info.visit_count] : 0,
            tab: param.topic_info.tab,
            boards: param.topic_info.boards ? [...param.topic_info.boards] : [],
            real_author_name: param.topic_info.real_author_name,
            author_id,
            author_name,
            author_avatar,
            source,
          }

          return topic;
        }).filter(v => v.title);
      this.logger.info(author_name, 'success...数量为:', totalTopics.length);
      return totalTopics;
    } catch (e) {
      console.error(e);
      throw e;
    }
  }


  /**
   * 全功能的http爬虫, 支持页面滚动
   */
  async commonFetch(param) {
    let totalTopics = [];

    const tab = param.topic_info.tab;

    let author_id;
    let author_name;
    let author_avatar;
    let source;

    if (param.author_info) {
      author_id = param.author_info.author_id;
      author_name = param.author_info.author_name;
      author_avatar = param.author_info.author_avatar;
      source = param.author_info.source || param.author_info.author_name + dayjs().format('YYYY-MM-DD');
    }

    let browser;
    let page;
    try {
      if (param.is_mobile) {
        [browser, page] = await this.ctx.helper.getMobileCanUseBrowserAndPage(param.url, param.selector_info.box, param.click, param.scroll, param.wait);
      } else {
        [browser, page] = await this.ctx.helper.getCanUseBrowserAndPage(param.url, param.selector_info.box, param.click, param.scroll, param.wait);
      }

      let eles;
      if (param.first_scroll) {
        if (param.scroll) {
          await this.ctx.helper.autoScrollByLimit(page, param.scroll);
        } else if (param.scroll_slow) {
          await this.ctx.helper.autoScrollByLimitSlow(page, param.scroll_slow);
        }
        eles = await page.$$(param.selector_info.box);
      } else {
        eles = await page.$$(param.selector_info.box);
        if (param.scroll) {
          await this.ctx.helper.autoScrollByLimit(page, param.scroll);
        } else if (param.scroll_slow) {
          await this.ctx.helper.autoScrollByLimitSlow(page, param.scroll_slow);
        }
      }

      if (param.top_selector_info) {
        let top_items = await page.$$(param.top_selector_info.box);
        for (let i = 0; i < top_items.length; i++) {
          let item = top_items[i];
          let title;
          let boards = param.topic_info.boards ? [...param.topic_info.boards] : [];
          let tags = param.topic_info.tags ? [...param.topic_info.tags] : [];
          if (await item.$(param.top_selector_info.title)) {
            title = await item.$eval(param.top_selector_info.title, v => v.innerText);
            if (!title) {
              title = await item.$eval(param.top_selector_info.title, v => v.title);
            }
            title = title.replace(/\s/g, '').trim();
          } else {
            let html = await item.evaluate(async el => {
              return el.outerHTML;
            }, item);
            console.error('没有当前元素:', html);
            continue;
          }
          let content;
          if (param.top_selector_info.content == '_root') {
            content = await item.getAttribute('data-golink');
          } else {
            content = await item.$eval(param.top_selector_info.content, v => v.href);
          }
          let pic;

          if (param.top_selector_info.pic && await item.$(param.top_selector_info.pic)) {
            pic = await item.$eval(param.top_selector_info.pic, v => v.src);
            if (!pic) {
              pic = await item.$eval(param.top_selector_info.pic, v => v.style['background-image']);
              pic = pic.replace(/url\("(.+)"\)/, '$1');
            }
            if (pic && pic.startsWith('data:image/')) {
              pic = undefined;
            }
          }


          let topic = {
            title,
            content,
            original_url: content,
            pic,
            tab,
            boards,
            tags,
            author_id,
            author_name,
            author_avatar,
            source,
          };
          totalTopics.push(topic);
        }
      }


      let cnt = eles.length;
      for (let i = 0; i < cnt; i++) {
        let el = eles[i];
        let title;
        let boards = param.topic_info.boards ? [...param.topic_info.boards] : [];
        let tags = param.topic_info.tags ? [...param.topic_info.tags] : [];
        if (await el.$(param.selector_info.title)) {
          title = await el.$eval(param.selector_info.title, v => v.innerText);
          if (!title) {
            title = await el.$eval(param.selector_info.title, v => v.title);
          }
          title = title.replace(/\s/g, '').trim();
        } else {
          let html = await el.evaluate(async el => {
            return el.outerHTML;
          }, el);
          console.error('没有当前元素:', html);
          continue;
        }
        let content;
        if (param.selector_info.content.includes('(')) {
          let reg_result = param.selector_info.content.match(/(.+)\((.+)\)/);
          let dom_name = reg_result[1];
          let dom_attr = reg_result[2];
          content = await el.$eval(dom_name, (v, arg) => v.getAttribute(arg), dom_attr);
        } else {
          if (param.selector_info.content == '_root') {
            content = await el.getAttribute('data-golink');
          } else {
            content = await el.$eval(param.selector_info.content, v => v.href);
          }
        }

        let pic;
        let pics;
        let intro;
        let visit_count = 0;
        let reply_count = 0;
        let real_author_name;
        let is_top = false;
        let is_hot = false;
        let is_new = false;
        let is_video = false;
        let is_advert = false;
        let video_length;
        let other;

        if (param.selector_info.pic && await el.$(param.selector_info.pic)) {
          pic = await el.$eval(param.selector_info.pic, v => v.src);
          if (!pic) {
            pic = await el.$eval(param.selector_info.pic, v => v.style['background-image']);
            pic = pic.replace(/url\("(.+)"\)/, '$1');
          }
          if (pic && pic.startsWith('data:image/')) {
            pic = undefined;
          }
        }

        if (param.selector_info.pics && await el.$(param.selector_info.pics)) {
          pics = await el.$$eval(param.selector_info.pics, vs => vs.map(v => v.src));
          if (pics.length > 0 && pics[0] == undefined) {
            pics = await el.$$eval(param.selector_info.pics, vs => vs.map(v => v.style['background-image']));
            pics = pics.map(pic => pic.replace(/url\("(.+)"\)/, '$1'));
          }
          pics = pics.slice(0, 5);
        }


        if (param.selector_info.intro && await el.$(param.selector_info.intro)) {
          intro = await el.$eval(param.selector_info.intro, v => v.innerText);
        }
        if (param.selector_info.tags && await el.$(param.selector_info.tags)) {
          let tmp_tags = await el.$$eval(param.selector_info.tags, vs => vs.map(v => v.innerText));
          tags = tags.concat(tmp_tags);
        }
        if (param.selector_info.visit_count && await el.$(param.selector_info.visit_count)) {
          visit_count = await el.$eval(param.selector_info.visit_count, v => v.innerText);
          visit_count = visit_count.trim();
          if (visit_count.endsWith('k')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
          } else if (visit_count.endsWith('K')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
          } else if (visit_count.endsWith('千')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
          } else if (visit_count.endsWith('w')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
          } else if (visit_count.endsWith('W')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
          } else if (visit_count.endsWith('万')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
          } else {
            visit_count = visit_count.replace(/[^\d]+/g, '');
            visit_count = +visit_count;
          }

          if (isNaN(visit_count)) {
            this.logger.error('非数字!!!');
            visit_count = 1;
          }
        }
        if (param.selector_info.real_author_name && await el.$(param.selector_info.real_author_name)) {
          real_author_name = await el.$eval(param.selector_info.real_author_name, v => v.innerText);
        }

        if (param.selector_info.is_top && await el.$(param.selector_info.is_top)) {
          is_top = true;
        }

        if (param.selector_info.is_hot && await el.$(param.selector_info.is_hot)) {
          is_hot = true;
        }

        if (param.selector_info.is_new && await el.$(param.selector_info.is_new)) {
          is_new = true;
        }

        if (param.selector_info.is_video && await el.$(param.selector_info.is_video)) {
          is_video = true;
        }

        if (param.selector_info.is_advert && await el.$(param.selector_info.is_advert)) {
          is_advert = true;
        }

        if (param.selector_info.other && await el.$(param.selector_info.other)) {
          other = await el.$eval(param.selector_info.other, v => v.innerText);
        }

        if (param.selector_info.video_length && await el.$(param.selector_info.video_length)) {
          video_length = await el.$eval(param.selector_info.video_length, v => v.innerText);
          is_video = true;
        }


        let topic = {
          title,
          content,
          original_url: content,
          pic,
          intro,
          visit_count,
          tab,
          boards,
          tags,
          real_author_name,
          is_top,
          is_hot,
          is_new,
          is_video,
          is_advert,
          video_length,
          other,
          author_id,
          author_name,
          author_avatar,
          source,
        };

        if (pics && pics.length > 2) {
          topic.pics = pics;
        }
        totalTopics.push(topic);
      }

      this.logger.info(author_name, param.topic_info && param.topic_info.boards, 'success...数量为:', totalTopics.length);
    } catch (e) {
      this.logger.error(e);
      await this.ctx.helper.dd_warn(`${author_name} ${e.message}`);
    } finally {
      if (browser) {
        await browser.close();
      }
      if (totalTopics.length == 0 && param.tab == 'yutang') {
        await this.ctx.helper.dd_warn(`注意注意, ${author_name} 本次commonFetch ${param.url} 数量为0`);
      }
      totalTopics = totalTopics.slice(0, 30);
      totalTopics.forEach(v => {
        if (v.pic && v.pic.startsWith('//')) {
          v.pic = 'https:' + v.pic;
        }
      });
      return totalTopics;
    }
  }


  async commonPodcastXimalaya(loginname) {
    const user = await this.ctx.model.User.findOne({loginname});
    const author_id = user.id;
    const author_name = loginname;
    const author_avatar = user.avatar_url;
    const source = author_name + dayjs().format('YYYY-MM-DD') + ':喜马拉雅';
    const url = this.ctx.helper.getFromUrl(user.froms, '喜马拉雅');


    const param = {
      url,
      author_info: {
        author_id,
        author_name,
        author_avatar,
        source
      },
      topic_info: {
        tab: 'podcast',
        boards: [...user.default_boards]
      },
      selector_info: {
        box: '.sound-list li',
        title: 'span.title._nO',
        content: '.text._nO a',
      }
    };

    const topics = await this.commonFetch(param);

    topics.forEach(t => {
      t.title = t.title.trim();
      t.title = '🎧 ' + t.title;
      t.tags.unshift('播客');
      t.pic = author_avatar;
    });

    this.ctx.helper.setDefaultPic(topics);
    return topics;
  }


  /**
   * 针对播客RSS的爬虫
   */
  async commonPodcastRss(loginname) {
    let topics = [];

    try {
      let tab = 'podcast';

      const user = await this.ctx.model.User.findOne({loginname})
      const author_id = user.id;
      const author_name = loginname;
      const author_avatar = user.avatar_url;
      const source = author_name + dayjs().format('YYYY-MM-DD') + ':播客RSS';
      const url = this.ctx.helper.getFromUrl(user.froms, '播客RSS');

      const default_boards = [...user.default_boards];

      let feed = await parser.parseURL(url);
      let items = feed.items;

      const max_len = 10;
      const cnt = items.length < max_len ? items.length : max_len;

      // 每次最多只录取最近10条
      for (let i = 0; i < cnt; i++) {
        let v = items[i];
        if (!v.link && !v.guid) {
          throw new Error(url + '没有link');
        }
        let pic = user.avatar_url;
        let intro = v.summary || (v.contentSnippet && v.contentSnippet.replace(/\n/g, ' ').substr(0, 50));
        if (intro) {
          intro += '...';
        } else {
          intro = undefined;
        }

        let topic = {
          title: '🎧 ' + v.title.trim(),
          content: v.link ? v.link.replace('www.ximalaya.com//', 'www.ximalaya.com/') : v.guid.replace('www.ximalaya.com//', 'www.ximalaya.com/'),
          original_url: v.link ? v.link.replace('www.ximalaya.com//', 'www.ximalaya.com/') : v.guid.replace('www.ximalaya.com//', 'www.ximalaya.com/'),
          intro,
          pic,
          boards: default_boards,
          tags: ['播客', ...default_boards],
          tab,
          author_id,
          author_name,
          author_avatar,
          source,
        };
        topics.push(topic);
      }
    } catch (e) {
      this.logger.error(loginname + 'commonPodcastRss 异常', e);
    } finally {
      topics = topics.slice(0, 30);
      this.ctx.helper.setDefaultPic(topics);
      return topics;
    }


  }


  /**
   * 针对小宇宙播客的爬虫
   */
  async commonPodcastXyz(loginname) {
    let topics = [];

    try {
      const user = await this.ctx.model.User.findOne({loginname});
      const author_id = user.id;
      const author_name = loginname;
      const author_avatar = user.avatar_url;
      const source = author_name + dayjs().format('YYYY-MM-DD') + ':中文播客榜';
      const url = this.ctx.helper.getFromUrl(user.froms, '中文播客榜');

      const param = {
        url,
        author_info: {
          author_id,
          author_name,
          author_avatar,
          source
        },
        topic_info: {
          tab: 'podcast',
          boards: [...user.default_boards]
        },
        selector_info: {
          box: 'ul.tab > li',
          title: '.title',
          content: 'a.card',
          pic: 'img.cover',
          intro: '.description',
          visit_count: '.stat:first-of-type'
        }
      };

      topics = await this.commonLight(param);


      topics.forEach(t => {
        t.title = t.title.trim();
        t.title = '🎧 ' + t.title;
        t.tags.unshift('播客');
      });


    } catch (e) {
      this.logger.error('commonPodcastXyz', e);
    } finally {
      this.ctx.helper.setDefaultPic(topics);
      return topics;
    }

  }


  /**
   * 针对小宇宙播客的爬虫(第一次导入需要, 带时间)
   */
  async commonPodcastXyzNew(loginname) {
    let topics = [];

    try {
      const user = await this.ctx.model.User.findOne({loginname});
      const author_id = user.id;
      const author_name = loginname;
      const author_avatar = user.avatar_url;
      const source = author_name + dayjs().format('YYYY-MM-DD') + ':中文播客榜';
      const url = this.ctx.helper.getFromUrl(user.froms, '中文播客榜');

      const param = {
        url,
        author_info: {
          author_id,
          author_name,
          author_avatar,
          source
        },
        topic_info: {
          tab: 'podcast',
          boards: [...user.default_boards]
        },
        selector_info: {
          box: 'ul.tab > li',
          title: '.title',
          content: 'a.card',
          pic: 'img.cover',
          intro: '.description',
          visit_count: '.stat:first-of-type',
          time: 'time'
        }
      };

      topics = await this.commonLight(param);


      topics.forEach(t => {
        t.title = t.title.trim();
        t.title = '🎧 ' + t.title;
        t.tags.unshift('播客');
      });


    } catch (e) {
      this.logger.error('commonPodcastXyz', e);
    } finally {
      this.ctx.helper.setDefaultPic(topics);
      return topics;
    }

  }




  /**
   * 针对百家号的爬虫
   */
  async commonBaijiahao(loginname, tab) {
    let topics = [];

    const max_retry_cnt = 3;
    let try_cnt = 0;
    while (topics.length == 0 && try_cnt < max_retry_cnt) {
      try {
        topics = await this._commonBaijiahao(loginname, tab);
      } catch (e) {
        this.logger.warn('百家号重试', loginname, try_cnt++);
      }
    }

    if (topics.length == 0) {
      await this.ctx.helper.dd_warn('百家号爬取失败: ' + loginname);
    }

    return topics;
  }

  async _commonBaijiahao(loginname, tab) {
    let totalTopics = [];
    const user = await this.ctx.model.User.findOne({loginname});
    if (!user) {
      throw new Error('commonBaijiahao参数异常');
    }

    const url = this.ctx.helper.getFromUrl(user.froms, '百家号新闻');
    if (!url) {
      throw new Error('commonBaijiahao参数异常');
    }

    const author_id = user.id;
    const author_name = user.loginname;
    const author_avatar = user.avatar_url;
    const source = author_name + dayjs().format('YYYY-MM-DD') + ':百家号新闻';


    let selector_info = {
      box: '.feed-item.pc-feed.tall-footer',
      title: '.text-title,.title',
      content: 'div',
      pic: '.s-image',
      pics: '.s-image',
    };

    const userDataDir = 'test-profile-dir';

    let context = await chromium.launchPersistentContext(userDataDir, {
      headless: false
    });
    try {
      let page = await context.newPage();

      await page.setExtraHTTPHeaders(
        {
          'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
        });
      await page.goto(url, {waitUntil: 'domcontentloaded', timeout: 10000});
      await page.waitForSelector(selector_info.box, {timeout: 10000});

      let eles = await page.$$(selector_info.box);
      let cnt = eles.length;

      for (let i = 0; i < cnt; i++) {
        const boards = [...user.default_boards] || [];
        let tags = [];

        let el = eles[i];

        let title;
        if (await el.$(selector_info.title)) {
          title = await el.$eval(selector_info.title, v => v.innerText);
          title = title.replace(/\n.+/g, '')
            .replace(/【(.+?)】(.+)?/, '$1')
            .replace(/「(.+?)」(.+)?/, '$1')
            .replace(/#(.+?)#/g, '$1')
            .trim();
        } else {
          let html = await el.evaluate(async v => {
            return v.outerHTML;
          });
          console.error('没有当前元素:', html);
          continue;
        }
        let content = await el.innerHTML();
        let regResult = content.match(/url="(.+?)"/);
        content = regResult[1];

        let pic;
        if (selector_info.pic && await el.$(selector_info.pic)) {
          pic = await el.$eval(selector_info.pic, v => v.src);
          if (!pic) {
            pic = await el.$eval(selector_info.pic, v => v.style['background-image']);
            pic = pic.replace(/url\("(.+)"\)/, '$1');
          }
          if (pic && pic.startsWith('data:image/')) {
            pic = undefined;
          }
        }

        let pics;
        if (selector_info.pics && await el.$(selector_info.pics)) {
          pics = await el.$$eval(selector_info.pics, vs => vs.map(v => v.src));
          if (pics.length > 0 && pics[0] == undefined) {
            pics = await el.$$eval(selector_info.pics, vs => vs.map(v => v.style['background-image']));
            pics = pics.map(pic => pic.replace(/url\("(.+)"\)/, '$1'));
          }
          pics = pics.slice(0, 5);
        }

        if (await el.$('.icon-play,.icon-video')) {
          boards.push('视频');
          tags.push('视频');
        }

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          tab,
          boards,
          tags,
          author_id,
          author_name,
          author_avatar,
          source
        };
        if (pics && pics.length > 2) {
          topic.pics = pics;
        }
        totalTopics.push(topic);
      }

      this._baijiahaoPostProcessing(totalTopics);
      return totalTopics;
    } catch (e) {
      throw e;
    } finally {
      if (context) {
        await context.close();
      }
    }
  }


  _baijiahaoPostProcessing(topics) {
    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (topic.pic) {
        topic.pic = topic.pic.replace(/JPEG@.+/, 'JPEG@s_0,w_500,q_90');
      }
      topic.content = topic.original_url = topic.content.replace(/&amp;/g, '&');
    }
  }


  async _tuicoolTryLogin() {
    const userDataDir = 'test-profile-dir';
    const context = await chromium.launchPersistentContext(userDataDir, {
      args: ['--disable-gpu', '--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36`],
      headless: true,
      ignoreHTTPSErrors: true,
    });

    try {
      const page = await context.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});

      const url = 'https://www.tuicool.com/user/3717797935';
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForTimeout(1000);
      const now_url = page.url();
      if (now_url == 'https://www.tuicool.com/login') {
        await page.waitForSelector('#xlEmail');
        await page.type('#xlEmail', 'yinhaixiang@qq.com');
        await page.type('#xlPassword', 'tuicool');
        await page.click('body > div.container-fluid > div.center_container.container-top > form > fieldset > div > div:nth-child(4) > button');
        await page.waitForTimeout(1000);
        this.logger.info('登录成功');
      } else {
        this.logger.info('已经登录过, 不需要再登录了');
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await context.close();
    }
  }


  /**
   * 轻量级的http爬虫
   */
  async commonLight(param) {
    let totalTopics = [];

    let tab = param.topic_info.tab;

    let author_id;
    let author_name;
    let author_avatar;
    let source;

    if (param.author_info) {
      author_id = param.author_info.author_id;
      author_name = param.author_info.author_name;
      author_avatar = param.author_info.author_avatar;
      source = param.author_info.source || param.author_info.author_name + dayjs().format('YYYY-MM-DD');
    }


    try {
      const response = await got({
        method: 'get',
        url: param.url,
      }, {
        https: {
          rejectUnauthorized: false
        }
      });

      const $ = cheerio.load(response.data);


      if (param.top_selector_info) {
        let top_items = $(param.top_selector_info.box).get();

        for (let i = 0; i < top_items.length; i++) {
          let item = top_items[i];
          let title;
          let content;
          let pic;
          let boards = param.topic_info.boards ? [...param.topic_info.boards] : [];
          let tags = param.topic_info.tags ? [...param.topic_info.tags] : [];

          title = $(item).find(param.top_selector_info.title).first().text().trim();
          if (!title) {
            continue;
          }

          if (param.top_selector_info.content) {
            content = url.resolve(param.url, $(item).find(param.top_selector_info.content).first().attr('href'));
          }

          if (param.top_selector_info.pic) {
            if ($(item).find(param.top_selector_info.pic).attr('src')) {
              pic = url.resolve(param.url, $(item).find(param.top_selector_info.pic).attr('src'));
            } else {
              pic = $(item).find(param.top_selector_info.pic).attr('style');
              if (pic) {
                pic = pic.replace(/.+url\((.+)\).+/, '$1');
              }
            }
          }

          let topic = {
            title,
            content,
            original_url: content,
            pic,
            tab,
            boards,
            tags,
            author_id,
            author_name,
            author_avatar,
            source,
          };
          totalTopics.push(topic);
        }
      }


      let items = $(param.selector_info.box).get();

      for (let i = 0; i < items.length; i++) {
        let item = items[i];
        let title;
        let content;
        let pic;
        let pics;
        let intro;
        let visit_count;
        let boards = param.topic_info.boards ? [...param.topic_info.boards] : [];
        let tags = param.topic_info.tags ? [...param.topic_info.tags] : [];
        let is_top = false;
        let is_hot = false;
        let is_video = false;
        let is_advert = false;
        let time;
        let other;
        let real_author_name;

        title = $(item).find(param.selector_info.title).first().text().trim();
        if (!title) {
          continue;
        }

        if (param.selector_info.content) {
          if (param.selector_info.content.includes('(')) {
            let reg_result = param.selector_info.content.match(/(.+)\((.+)\)/);
            let dom_name = reg_result[1];
            let dom_attr = reg_result[2];
            try {
              content = url.resolve(param.url, $(item).find(dom_name).attr(dom_attr));
            } catch (e) {
              continue;
            }
          } else {
            content = url.resolve(param.url, $(item).find(param.selector_info.content).first().attr('href'));
          }
        }

        if (param.selector_info.pic) {
          if ($(item).find(param.selector_info.pic).attr('data-src')) {
            pic = url.resolve(param.url, $(item).find(param.selector_info.pic).attr('data-src'));
          } else if ($(item).find(param.selector_info.pic).attr('src')) {
            pic = url.resolve(param.url, $(item).find(param.selector_info.pic).attr('src'));
          } else {
            pic = $(item).find(param.selector_info.pic).attr('style');
            if (pic) {
              pic = pic.replace(/.+url\((.+)\).+/, '$1');
            }
          }
        }

        if (param.selector_info.pics) {
          pics = $(item).find(param.selector_info.pics).map((i, el) => {
            return $(el).attr('src');
          }).get();
          pics = pics.slice(0, 5);
        }


        if ($(item).find(param.selector_info.intro).text()) {
          intro = $(item).find(param.selector_info.intro).text().trim();
          intro = intro.replace(/\n/g, ' ');
        }
        if ($(item).find(param.selector_info.tags).text()) {
          let tags_tmp = $(item).find(param.selector_info.tags).map((i, el) => {
            return $(el).text();
          }).get();
          tags = tags.concat(tags_tmp);
        }
        if ($(item).find(param.selector_info.visit_count).text()) {
          visit_count = $(item).find(param.selector_info.visit_count).text();
          if (visit_count) {
            visit_count = visit_count.trim();
            if (visit_count.endsWith('k')) {
              visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
            } else if (visit_count.endsWith('K')) {
              visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
            } else if (visit_count.endsWith('千')) {
              visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
            } else if (visit_count.endsWith('w')) {
              visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
            } else if (visit_count.endsWith('W')) {
              visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
            } else if (visit_count.endsWith('万')) {
              visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
            } else {
              visit_count = visit_count.replace(/[^\d]+/g, '');
              visit_count = +visit_count;
            }
          }
        }

        if (!visit_count) {
          visit_count = 0;
        }


        if (param.selector_info.is_top) {
          if ($(item).find(param.selector_info.is_top)) {
            if ($(item).find(param.selector_info.is_top).html() != null) {
              is_top = true;
            }
          }
        }

        if (param.selector_info.is_hot) {
          if ($(item).find(param.selector_info.is_hot)) {
            if ($(item).find(param.selector_info.is_hot).html() != null) {
              is_hot = true;
            }
          }
        }

        if (param.selector_info.is_video) {
          if ($(item).find(param.selector_info.is_video)) {
            if ($(item).find(param.selector_info.is_video).html() != null) {
              is_video = true;
            }
          }
        }

        if (param.selector_info.is_advert) {
          if ($(item).find(param.selector_info.is_advert)) {
            if ($(item).find(param.selector_info.is_advert).html() != null) {
              is_advert = true;
            }
          }
        }

        if (param.selector_info.time) {
          if ($(item).find(param.selector_info.time)) {
            time = $(item).find(param.selector_info.time).attr('datetime');
          }
        }

        if (param.selector_info.other) {
          if ($(item).find(param.selector_info.other)) {
            if ($(item).find(param.selector_info.other).html() != null) {
              other = $(item).find(param.selector_info.other).text().trim();
            }
          }
        }

        if (param.selector_info.real_author_name) {
          if ($(item).find(param.selector_info.real_author_name)) {
            if ($(item).find(param.selector_info.real_author_name).html() != null) {
              real_author_name = $(item).find(param.selector_info.real_author_name).text().trim();
            }
          }
        }


        let topic = {
          title,
          content,
          original_url: content,
          pic,
          intro,
          visit_count,
          tab,
          boards,
          tags,
          is_top,
          is_hot,
          is_video,
          is_advert,
          time,
          other,
          real_author_name,
          author_id,
          author_name,
          author_avatar,
          source,
        };
        if (pics && pics.length > 2) {
          topic.pics = pics;
        }
        totalTopics.push(topic);
      }

      // 如果有detail字段, 代表要爬取详情页的信息
      if (param.selector_info.detail) {
        totalTopics = await this.service.topicSvc.filterExistTopics(totalTopics);

        totalTopics = await Promise.all(
          totalTopics.map(async (item) => {
            const detailResponse = await got({
              method: 'get',
              url: item.content,
            });

            const $ = cheerio.load(detailResponse.data);

            for (const key in param.selector_info.detail) {
              if (!item[key]) {
                let $selector = $(param.selector_info.detail[key]);
                let value;
                if (key == 'pic') {
                  value = $selector.attr('src');
                  value = url.resolve(param.url, value);
                } else if (key == 'content') {
                  value = $selector.attr('href');
                  value = url.resolve(param.url, value);
                }
                item[key] = value;
              }
            }
            return item;
          })
        );
      }
      this.logger.info(author_name, param.topic_info && param.topic_info.boards, 'success...数量为:', totalTopics.length);
    } catch (e) {
      this.logger.error(e);
      await this.ctx.helper.dd_warn(`${author_name} ${e.message}`);
    } finally {
      if (totalTopics.length == 0 && param.tab == 'yutang') {
        await this.ctx.helper.dd_warn(`注意注意, ${author_name} 本次commonLight ${param.url} 数量为0`);
      }
      return totalTopics;
    }
  }


  async loginData258(context) {
    const page = await context.newPage();
    await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});

    const login_url = 'https://mp.data258.com/user/login';
    await page.goto(login_url, {waitUntil: 'domcontentloaded'});
    await page.waitForTimeout(5000);
    await page.screenshot({path: 'screenshot/screenshot.png'});

    const file_base64 = fs.readFileSync('./screenshot.png', 'base64');
    this.logger.info('截图成功');
    const formData = {
      'user': 'yinhaixiang',
      'pass': 'sean0410',
      'codetype': '1902',
      'file_base64': file_base64
    };

    const response = await got({
      method: 'post',
      url: 'http://upload.chaojiying.net/Upload/Processing.php',
      headers: {
        "content-type": "application/x-www-form-urlencoded",
      },
      timeout: 60000,
      body: JSON.stringify(formData)
    });

    const code = response.data.pic_str;
    this.logger.info('获取验证码成功:', code);

    await page.type('#L_user', 'yinhaixiang');
    await page.waitForTimeout(500);
    await page.type('#L_pass', 'sean0410');
    await page.waitForTimeout(500);
    await page.type('#L_vercode', code);
    await page.waitForTimeout(500);
    await page.click('.layui-btn');
    await page.waitForTimeout(2000);
    this.logger.info('data258 登录成功');
    await page.close();
  }


  async loginData258New() {
    const userDataDir = 'test-profile-dir';
    const context = await firefox.launchPersistentContext(userDataDir, {
      args: ['--disable-gpu', '--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36`],
      headless: true,
      ignoreHTTPSErrors: true,
    });
    const page = await context.newPage({
      proxy: {
        server: 'http://u5329.b5.t.16yun.cn:6460',
        username: '16GADNHJ',
        password: '936441'
      }
    });
    await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});

    try {
      const login_url = 'https://mp.data258.com/user/login';
      await page.goto(login_url, {waitUntil: 'domcontentloaded'});

      await page.waitForTimeout(2000);
      await page.screenshot({path: 'screenshot/screenshot.png'});
      const file_base64 = fs.readFileSync('./screenshot.png', 'base64');
      this.logger.info('截图成功');
      const formData = {
        'user': 'yinhaixiang',
        'pass': 'sean0410',
        'codetype': '1902',
        'file_base64': file_base64
      };
      const response = await got({
        method: 'post',
        url: 'http://upload.chaojiying.net/Upload/Processing.php',
        headers: {
          "content-type": "application/x-www-form-urlencoded",
        },
        timeout: 60000,
        body: JSON.stringify(formData)
      });

      console.log(response.data);
      const code = response.data.pic_str;
      this.logger.info('获取验证码成功:', code);

      await page.waitForSelector('#L_user', {timeout: 2000});
      await page.type('#L_user', 'yinhaixiang');
      await page.waitForTimeout(500);
      await page.type('#L_pass', 'sean0410');
      await page.waitForTimeout(500);
      await page.type('#L_vercode', code);
      await page.waitForTimeout(500);
      await page.click('.layui-btn');
      await page.waitForTimeout(2000);
      let now_url = page.url();
      this.logger.warn('尝试登录后的当前url:', now_url);
      if (now_url != 'https://mp.data258.com/user/login') {
        this.logger.info('data258 登录成功');
      } else {
        this.logger.info('data258 登录失败');

        const errorformData = {
          'user': 'yinhaixiang',
          'pass': 'sean0410',
          'id': response.data.pic_id,
        };
        let errorResponse = await got({
          method: 'post',
          url: 'http://upload.chaojiying.net/Upload/ReportError.php',
          headers: {
            "content-type": "application/x-www-form-urlencoded",
          },
          timeout: 60000,
          body: JSON.stringify(errorformData)
        });
        this.logger.warn('报错后返回:', errorResponse.data);
      }
    } catch (e) {
      this.logger.error(e);
      await page.screenshot({path: 'screenshot/jt-error-login.png'});
    } finally {
      await context.close();
    }
  }


}

module.exports = CommonSvc;
